f (!empty($uidarr)) { $uids = array(); $update = array(); foreach ($uidarr as $_uid => $n) { $uids[] = $_uid; $update[$_uid] = array('articles-' => $n); 'mysql' != $conf['cache']['type'] and cache_delete('user-' . $_uid); } user_big_update(array('uid' => $uids), $update); } !empty($operate_create) && function_exists('operate_big_insert') and operate_big_insert($operate_create); return TRUE; } // 大数据量容易超时 删除用户时使用，删除主题 回复 栏目统计 附件 全站统计 function well_thread_delete_all_by_uid($uid) { // 用户主题数 $user = user_read_cache($uid); set_time_limit(0); // 删除所有回复 if ($user['comments']) { $postist = comment_pid_find_by_uid($uid, 1, $user['comments'], FALSE); $pidarr = array(); foreach ($postist as $val) { $pidarr[] = $val['pid']; } unset($postist); !empty($pidarr) and comment_delete_by_pids($pidarr); } if ($user['articles']) { // 如果主题、附件和回复数量太大会超时 $tidlist = thread_tid_find_by_uid($uid, 1, $user['articles'], FALSE, 'tid', array('tid')); $tidarr = array(); foreach ($tidlist as $val) { $tidarr[] = $val['tid']; } unset($tidlist); !empty($tidarr) and well_thread_delete_all($tidarr); } return TRUE; } // 搜索标题 function well_thread_find_by_keyword($keyword, $d = NULL) { if (empty($keyword)) return NULL; $db = $_SERVER['db']; $d = $d ? $d : $db; if (!$d) return FALSE; $threadlist = db_sql_find("SELECT * FROM `{$d->tablepre}website_thread` WHERE subject LIKE '%$keyword%' LIMIT 60;", 'tid', $d); if ($threadlist) { $threadlist = arrlist_multisort($threadlist, 'tid', FALSE); foreach ($threadlist as &$thread) { well_thread_format($thread); // 关键词标色 //$thread['subject'] = comment_highlight_keyword($thread['subject'], $keyword); } } return $threadlist; } // 查找 最后评论 lastpid function well_thread_find_lastpid($tid) { $arr = comment_pid_read(array('tid' => $tid), array('pid' => -1), array('pid')); $lastpid = empty($arr) ? 0 : $arr['pid']; return $lastpid; } // 更新最后的 uid function well_thread_update_last($tid) { if (empty($tid)) return FALSE; $lastpid = well_thread_find_lastpid($tid); if (empty($lastpid)) return FALSE; $lastpost = comment_read($lastpid); if (empty($lastpost)) return FALSE; $r = well_thread_update($tid, array('lastuid' => $lastpost['uid'])); return $r; } function well_thread_maxid() { $n = db_maxid('website_thread', 'tid'); return $n; } // 主题状态 0:通过 1~9 审核:1待审核 10~19:10退稿 11逻辑删除 function well_thread_format(&$thread) { global $gid, $uid, $forumlist; $conf = _SERVER('conf'); if (empty($thread)) return; $thread['create_date_fmt'] = humandate($thread['create_date']); $thread['last_date_fmt'] = humandate($thread['last_date']); $thread['create_date_fmt_ymd'] = date('Y-m-d', $thread['create_date']); $thread['last_date_fmt_ymd'] = date('Y-m-d', $thread['last_date']); $user = user_read_cache($thread['uid']); $onlinelist = online_user_list_cache(); $user['online_status'] = isset($onlinelist[$user['uid']]) ? 1 : 0; $thread['username'] = $user['username']; $thread['user_avatar_url'] = array_value($user, 'avatar_url'); $thread['user'] = user_safe_info($user); unset($user); $forum = array_value($forumlist, $thread['fid']); $thread['forum_name'] = array_value($forum, 'name'); $thread['forum_url'] = array_value($forum, 'url'); if ($thread['last_date'] == $thread['create_date']) { $thread['last_date_fmt'] = ''; $thread['lastuid'] = 0; $thread['lastusername'] = ''; } else { $lastuser = $thread['lastuid'] ? user_read_cache($thread['lastuid']) : array(); $thread['lastusername'] = $thread['lastuid'] ? $lastuser['username'] : lang('guest'); $thread['lastuser'] = $thread['lastuid'] ? user_safe_info($lastuser) : array(); unset($lastuser); } $thread['url'] = url('read-' . $thread['tid'], '', FALSE); if ($conf['url_rewrite_on'] > 1) { !empty($forum['well_alias']) and $thread['url'] = url(urlencode($forum['well_alias'])."-$thread[create_date]a$thread[tid]", '', FALSE); } else { $thread['url'] = url("read-$thread[create_date]a$thread[tid]", '', FALSE); } $thread['user_url'] = url('user-' . $thread['uid']); $thread['sticky_class'] = ''; if ($thread['sticky'] > 0) { if (1 == $thread['sticky']) { $thread['sticky_class'] = 'success'; } elseif (2 == $thread['sticky']) { $thread['sticky_class'] = 'warning'; } elseif (3 == $thread['sticky']) { $thread['sticky_class'] = 'danger'; } } $nopic = view_path() . 'img/nopic.png'; if ($thread['icon']) { $attach_dir_save_rule = array_value($conf, 'attach_dir_save_rule', 'Ym'); $day = date($attach_dir_save_rule, $thread['icon']); $thread_format_icon_default = 1; if (1 == $thread_format_icon_default) { // 本地文件绝对路径 $destfile = $conf['upload_path'] . 'thumbnail/' . $day . '/' . $thread['uid'] . '_' . $thread['tid'] . '_' . $thread['icon'] . '.jpeg'; // 本地 $thread['icon_fmt'] = is_file($destfile) ? file_path($thread['attach_on']) . 'thumbnail/' . $day . '/' . $thread['uid'] . '_' . $thread['tid'] . '_' . $thread['icon'] . '.jpeg' : $nopic; if (1 == $conf['attach_on'] && 1 == $thread['attach_on']) { // 云储存 $thread['icon_fmt'] = file_path($thread['attach_on']) . 'thumbnail/' . $day . '/' . $thread['uid'] . '_' . $thread['tid'] . '_' . $thread['icon'] . '.jpeg'; } elseif (2 == $conf['attach_on'] && 2 == $thread['attach_on']) { // 图床 未上传成功 本地图片在的话使用本地，不在则默认 $thread['icon_fmt'] = $thread['image_url'] ? $thread['image_url'] : $thread['icon_fmt']; } } } else { $thread['icon_fmt'] = $nopic; } // 回复页面 $thread['pages'] = ceil($thread['posts'] / $conf['comment_pagesize']); $thread['tag_fmt'] = $thread['tag'] ? xn_json_decode($thread['tag']) : ''; // 权限判断 $thread['allowupdate'] = ($uid == $thread['uid']) || forum_access_mod($thread['fid'], $gid, 'allowupdate'); $thread['allowdelete'] = (group_access($gid, 'allowuserdelete') and $uid == $thread['uid']) || forum_access_mod($thread['fid'], $gid, 'allowdelete'); $thread['allowtop'] = forum_access_mod($thread['fid'], $gid, 'allowtop'); $thread = well_thread_safe_info($thread); } function well_thread_format_last_date(&$thread) { if ($thread['last_date'] != $thread['create_date']) { $thread['last_date_fmt'] = humandate($thread['last_date']); } else { $thread['create_date_fmt'] = humandate($thread['create_date']); } } // 对 $threadlist 权限过滤 function well_thread_list_access_filter(&$threadlist, $gid) { global $forumlist; if (empty($threadlist)) return NULL; foreach ($threadlist as $tid => $thread) { if (empty($forumlist[$thread['fid']]['accesson'])) continue; if ($thread['sticky'] > 0) continue; if (!forum_access_user($thread['fid'], $gid, 'allowread')) { unset($threadlist[$tid]); } } } function well_thread_safe_info($thread) { unset($thread['userip'], $thread['user']['threads'], $thread['user']['posts'], $thread['user']['credits'], $thread['user']['golds'], $thread['user']['money']); empty($thread['user']) || $thread['user'] = user_safe_info($thread['user']); return $thread; } // 过滤安全数据 function well_thread_filter(&$val) { unset($val['userip'], $val['fid'], $val['flagid'], $val['type'], $val['user'], $val['create_date']); } //------------------------ 其他方法 ------------------------ // 集合主题tid，统一拉取，避免多次查询thread表 function thread_unified_pull($arr) { global $gid, $fid; $stickylist = array_value($arr, 'stickylist', array()); $tidlist = array_value($arr, 'tidlist', array()); //$fid = array_value($arr, 'fid'); // 合并过滤空数组 //$tidlist = array_filter($stickylist + $tidlist); $tidarrlist = $tidlist = $stickylist + $tidlist; // 版块自定义 list($flaglist, $flagtids) = flag_thread_by_fid($fid); empty($flagtids) || $tidarrlist += $flagtids; unset($flagtids); // 在这之前合并所有二维数组 tid值为键/array('tid值' => tid值) $tidarr = empty($tidarrlist) ? array() : arrlist_values($tidarrlist, 'tid'); // 在这之前使用array_merge()前合并所有一维数组 tid/array(1,2,3) if (empty($tidarr)) return NULL; $tidarr = array_unique($tidarr); // 主题相关统一遍历后再归类 $arrlist = well_thread_find($tidarr, count($tidarr)); // 过滤没有权限访问的主题 / filter no permission thread well_thread_list_access_filter($arrlist, $gid); $threadlist = array(); foreach ($arrlist as $_tid => &$_thread) { $_thread = well_thread_safe_info($_thread); // 归类列表数据 isset($tidlist[$_thread['tid']]) and $threadlist[$_tid] = $_thread; // flag thread if (!empty($flaglist)) { foreach ($flaglist as $key => $val) { if (isset($val['tids']) && in_array($_thread['tid'], $val['tids'])) { $flaglist[$key]['list'][array_search($_thread['tid'], $val['tids'])] = $_thread; ksort($flaglist[$key]['list']); } } } } // 按之前tidlist排序 $threadlist = array2_sort_key($threadlist, $tidlist, 'tid'); unset($arrlist, $tidlist); $arr = array('threadlist' => $threadlist, 'flaglist' => $flaglist); return $arr; } // read.php 详情页其他主题调用，集合tid统一拉取数据，最后归类 function thread_other_pull($thread) { global $forumlist, $gid; $fid = array_value($thread, 'fid'); $forum = array_value($forumlist, $fid); if (empty($forum)) return NULL; //$tid = array_value($thread, 'tid'); //$tag_fmt = array_value($thread, 'tag_fmt'); $arrlist = array(); $tidlist = array(); // 版块自定义 list($flaglist, $flagtids) = flag_thread_by_fid($fid); empty($flagtids) || $tidlist += $flagtids; unset($flagtids); // 在这之前合并所有二维数组 tid值为键/array('tid值' => tid值) $tidarr = empty($tidlist) ? array() : arrlist_values($tidlist, 'tid'); // 在这之前使用array_merge()前合并所有一维数组 tid/array(1,2,3) if (empty($tidarr)) return NULL; $tidarr = array_unique($tidarr); // 主题相关统一遍历后再归类 $threadlist = well_thread_find($tidarr, count($tidarr)); // 过滤没有权限访问的主题 / filter no permission thread well_thread_list_access_filter($threadlist, $gid); foreach ($threadlist as &$_thread) { $_thread = well_thread_safe_info($_thread); // flag thread if (!empty($flaglist)) { foreach ($flaglist as $key => $val) { if (isset($val['tids']) && in_array($_thread['tid'], $val['tids'])) { $flaglist[$key]['list'][array_search($_thread['tid'], $val['tids'])] = $_thread; ksort($flaglist[$key]['list']); } } } } unset($threadlist); if (!empty($flaglist)) { foreach ($flaglist as &$val) { $i = 0; if (!isset($val['list'])) continue; foreach ($val['list'] as &$v) { ++$i; $v['i'] = $i; } } $arrlist['flaglist'] = $flaglist; unset($flaglist); } return $arrlist; } //--------------------------cache-------------------------- // 已格式化 从缓存中读取，避免重复从数据库取数据 function well_thread_read_cache($tid) { global $conf; $key = 'website_thread_' . $tid; static $cache = array(); // 用静态变量只能在当前 request 生命周期缓存，跨进程需要再加一层缓存：redis/memcached/xcache/apc if (isset($cache[$key])) return $cache[$key]; if ('mysql' == $conf['cache']['type']) { $r = well_thread_read($tid); } else { $r = cache_get($key); if (NULL === $r) { $r = well_thread_read($tid); $r and cache_set($key, $r, 1800); } } $cache[$key] = $r ? $r : NULL; return $cache[$key]; } ?>爬虫
首页 >  标签 >  爬虫

推荐几款优秀的网络爬虫工具

1.前言 一、 什么是网络爬虫？ 网络爬虫又名“网络蜘蛛”，是通过网页的链接地址来寻找网页，从网站某一个页面开始，读取网页的内容&#xff0

7月前1160

探寻爬虫世界01：HTML页面结构

文章目录 一、引言 （一）背景介绍：选择爬取51job网站数据的原因 （二）目标与需求明确：爬取51job网站数据的目的与用户需求 二、网页结构探索 （一）51job网页结构分析 1、页面组成：了解51job网站的整体结构 2、页面元素：探

7月前830

python爬虫 爬取网页图片

目录 一：爬虫基础 二：安装html解析的python工具 三：爬取网页图片 一：爬虫基础 爬虫基本过程： 1.请求标头 h

7月前960

Python爬虫突破某360查询网站反爬验证码

某360查询网址同一ip频繁访问的话会弹出验证码， 该验证码特点：get获取的验证码图片和网页展现的不一致（所以无法通过图像识别破解）；该验证码是针对ip，输入验证码后会给出一个大概一天有效期的cookie，在此阶段可以大量频繁访问了。

7月前1120

爬虫漫游指南：无头浏览器puppeteer的检测攻防

1. 引言 许多爬虫初学者在接触到无头浏览器的时候都会有一种如获至宝的感觉，仿佛看到了爬虫的终极解决方案。无论是所有爬虫教程中都会出现的PhantomJS、Selenium，亦或是相对冷门的Nightmare，到后来居上的Puppetee

8月前1120

爬虫进阶路程5——无头浏览器的坑

本来以为经历前面四道坑，算是走到了反爬的顶端，没想到不到三个月再次倒下了，因为之前的爬虫程序突然被反爬了，怎都拿不到数据，一开始以

8月前960

爬虫实战 -- QQ空间自动点赞！太强了呀！

前景提要 因为我周围的小伙伴们天天跟我说的最多的一句话就是：空间第一条点赞。 所以说我还不如直接做一个自动点赞的代码呢，免得天天催我点赞。 PS：如有需要Python学习

8月前650

输入qq号强制进空间_Python爬虫实战：QQ空间全自动点赞工具

QQ空间自动点赞前景提要目标确定分析介绍登陆获取cookie寻找XML寻找可变参数获取第一个空间动态寻找点赞所需的URL寻找可变参数功能提升到秒赞全部代码最后还是希望你们能给我点一波小小的关注。奉上自己诚挚的爱心私信小编01即可获取大量Py

8月前960

python 爬虫之路教程

原址摘要： From：https:piaosanlang.gitbooks.iospiderscontent 爬虫入门初级篇 IDE 选择： PyCharm(推

8月前780

python爬虫之scrapy

scrapy安装 在windows下，在dos中运行pip install Scrapy报错 采用pip安装，安装时可能会出现安装错误Microsoft Visual C 14.0 is re

8月前1290

python 爬虫基础学习

原址摘要： From：https:piaosanlang.gitbooks.iospiderscontent 爬虫入门初级篇 IDE 选择： PyCharm(推

8月前1130